{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将所有特征串联起来，构成RS_Train.csv\n",
    "#RS_Test.csv\n",
    "#为最后推荐系统做准备\n",
    "from __future__ import division\n",
    "\n",
    "import cPickle\n",
    "import numpy as np\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "from numpy.random import random  \n",
    "from collections import defaultdict\n",
    "import pandas as pd \n",
    "\n",
    "class RecommonderSystem:\n",
    "  def __init__(self, trainfile):\n",
    "    # 读入数据做初始化\n",
    "    \n",
    "    #用户和活动新的索引\n",
    "    self.userIndex = cPickle.load(open(\"PE_userIndex.pkl\", 'rb'))\n",
    "    self.eventIndex = cPickle.load(open(\"PE_eventIndex.pkl\", 'rb'))\n",
    "    self.n_users = len(self.userIndex)\n",
    "    self.n_items = len(self.eventIndex)\n",
    "    \n",
    "    #用户-活动关系矩阵R\n",
    "    #在train_SVD会重新从文件中读取,二者要求的格式不同，来不及统一了\n",
    "    self.userEventScores = sio.mmread(\"PE_userEventScores\").todense()\n",
    "    \n",
    "    #倒排表\n",
    "    ##每个用户参加的事件\n",
    "    self.itemsForUser = cPickle.load(open(\"PE_eventsForUser.pkl\", 'rb'))\n",
    "    ##事件参加的用户\n",
    "    self.usersForItem = cPickle.load(open(\"PE_usersForEvent.pkl\", 'rb'))\n",
    "    \n",
    "    #基于模型的协同过滤参数初始化,训练\n",
    "    self.init_Data(trainfile)\n",
    "    self.train_SVD()\n",
    "    \n",
    "    #根据用户属性计算出的用户之间的相似度\n",
    "    self.userSimMatrix = sio.mmread(\"US_userSimMatrix\").todense()\n",
    "    \n",
    "    #根据活动属性计算出的活动之间的相似度\n",
    "    self.eventPropSim = sio.mmread(\"EV_eventPropSim\").todense()\n",
    "    self.eventContSim = sio.mmread(\"EV_eventContSim\").todense()\n",
    "    \n",
    "    #每个用户的朋友的数目\n",
    "    self.numFriends = sio.mmread(\"UF_numFriends\")\n",
    "    #用户的每个朋友参加活动的分数对该用户的影响\n",
    "    self.userFriends = sio.mmread(\"UF_userFriends\").todense()\n",
    "    \n",
    "    #活动本身的热度\n",
    "    self.eventPopularity = sio.mmread(\"EA_eventPopularity\").todense()\n",
    "\n",
    "  def init_Data(self, trainfile, K=20):\n",
    "    #初始化模型参数（for 基于模型的协同过滤SVD_CF）\n",
    "    self.K = K  \n",
    "    \n",
    "    #init train data\n",
    "    self.X = pd.read_csv(trainfile).values\n",
    "    self.bu = {}\n",
    "    self.bi = {}\n",
    "    self.P = {}\n",
    "    self.Q = {}\n",
    "    self.itemRatesForUser = {}\n",
    "    self.myUsersForItem = {}\n",
    "    \n",
    "    for x in self.X:\n",
    "        u = x[0]\n",
    "        i = x[1]\n",
    "        s = x[4]\n",
    "        \n",
    "        self.bu.setdefault(u, 0)\n",
    "        self.bi.setdefault(i, 0)\n",
    "        self.P.setdefault(u, random(self.K)/10*(np.sqrt(self.K)))\n",
    "        self.Q.setdefault(i, random(self.K)/10*(np.sqrt(self.K)))\n",
    "        \n",
    "        self.itemRatesForUser.setdefault(u, {})\n",
    "        self.itemRatesForUser[u][i] = s\n",
    "        \n",
    "        self.myUsersForItem.setdefault(i, [])\n",
    "        self.myUsersForItem[i].append(u)\n",
    "    \n",
    "    self.userSims = {}\n",
    "    for i in self.myUsersForItem:\n",
    "        users = self.myUsersForItem[i]\n",
    "        for u1 in users:\n",
    "            self.userSims.setdefault(u1, {})\n",
    "            for u2 in users:\n",
    "                self.userSims.setdefault(u2, {})\n",
    "                self.userSims[u1][u2] = \"null\"\n",
    "                self.userSims[u2][u1] = \"null\"\n",
    "                \n",
    "    self.itemSims = {}\n",
    "    \n",
    "  def train_SVD(self,steps=100,gamma=0.04,Lambda=0.15):\n",
    "    #训练SVD模型（for 基于模型的协同过滤SVD_CF）\n",
    "    #gamma：为学习率\n",
    "    #Lambda：正则参数\n",
    "    \n",
    "    #偷懒了，为了和原来的代码的输入接口一样，直接从训练文件中去读取数据\n",
    "    print \"SVD Train...\"\n",
    "    self.mu = 0.0\n",
    "    n_records = 0\n",
    "    \n",
    "    for x in self.X:\n",
    "        self.mu += x[4]\n",
    "        n_records += 1\n",
    "    self.mu /= n_records\n",
    "    \n",
    "    # 请补充完整SVD模型训练过程\n",
    "    \n",
    "    for step in range(steps):\n",
    "        print 'the ',step,'-th  step is running'\n",
    "        kk = np.random.permutation(self.X.shape[0])\n",
    "        for j in range(self.X.shape[0]):\n",
    "            x = self.X[j]\n",
    "            u = x[0]\n",
    "            i = x[1]\n",
    "            eui = x[4] - self.pred_SVD(u, i)\n",
    "            self.bu[u] -= gamma * (-eui + Lambda * self.bu[u])\n",
    "            self.bi[i] -= gamma * (-eui + Lambda * self.bi[i])\n",
    "            for k in range(self.K):\n",
    "                puk = self.P[u][k]\n",
    "                qki = self.Q[i][k]\n",
    "                self.P[u][k] -= gamma * ( -eui * qki + Lambda * puk)\n",
    "                self.Q[i][k] -= gamma * ( -eui * puk + Lambda * qki)\n",
    "        gamma*=0.93\n",
    "    \n",
    "    print \"SVD trained\"\n",
    "\n",
    "  def pred_SVD(self, uid, i_id):\n",
    "    #根据当前参数，预测用户uid对Item（i_id）的打分   \n",
    "    if uid not in self.bu or i_id not in self.bi:\n",
    "        return 0\n",
    "    ans=self.mu + self.bi[i_id] + self.bu[uid] + np.sum(self.P[uid]*self.Q[i_id])\n",
    "\n",
    "    #将打分范围控制在0-1之间\n",
    "    if ans>1:  \n",
    "        return 1  \n",
    "    elif ans<0:  \n",
    "        return 0\n",
    "    return ans  \n",
    "\n",
    "  def sim_cal_UserCF(self, uid1, uid2):\n",
    "    #请补充基于用户的协同过滤中的两个用户uid1和uid2之间的相似度（根据两个用户对item打分的相似度）\n",
    "    \n",
    "    if self.userSims[uid1][uid2] != \"null\":\n",
    "        return self.userSims[uid1][uid2]\n",
    "    \n",
    "    itemRates1 = self.itemRatesForUser[uid1]\n",
    "    itemRates2 = self.itemRatesForUser[uid2]\n",
    "    \n",
    "    m1 = 1.0 * np.sum(itemRates1.values()) / len(itemRates1)\n",
    "    m2 = 1.0 * np.sum(itemRates2.values()) / len(itemRates2)\n",
    "    \n",
    "    #分子\n",
    "    upsum = 0\n",
    "    #分母\n",
    "    dwsum1 = 0\n",
    "    dwsum2 = 0\n",
    "    \n",
    "    for item1 in itemRates1:\n",
    "        if item1 in itemRates2:\n",
    "            r1 = itemRates1[item1]\n",
    "            r2 = itemRates2[item1]\n",
    "            upsum += (r1 - m1) * (r2 - m2)\n",
    "            dwsum1 += (r1 - m1)**2\n",
    "            dwsum2 += (r2 - m2)**2\n",
    "            \n",
    "    if dwsum1 == 0 or dwsum2 == 0:\n",
    "        self.userSims[uid1][uid2] = 0\n",
    "        self.userSims[uid2][uid1] = 0\n",
    "        return 0\n",
    "    \n",
    "    similarity = 1.0 *  upsum / (np.sqrt(dwsum1) *  np.sqrt(dwsum2))\n",
    "    \n",
    "    self.userSims[uid1][uid2] = similarity\n",
    "    self.userSims[uid2][uid1] = similarity\n",
    "    \n",
    "    return similarity  \n",
    "\n",
    "  def userCFReco(self, userId, eventId):\n",
    "    \"\"\"\n",
    "    根据User-based协同过滤，得到event的推荐度\n",
    "    基本的伪代码思路如下：\n",
    "    for item i\n",
    "      for every other user v that has a preference for i\n",
    "        compute similarity s between u and v\n",
    "        incorporate v's preference for i weighted by s into running aversge\n",
    "    return top items ranked by weighted average\n",
    "    \"\"\"\n",
    "    #请补充完整代码\n",
    "    if userId not in self.itemRatesForUser:\n",
    "        return 0.0\n",
    "    itemRates = self.itemRatesForUser[userId]\n",
    "    users = self.myUsersForItem[eventId]\n",
    "    if len(itemRates) == 0 or len(users) <= 1:\n",
    "        return 0.0\n",
    "    \n",
    "    mu = 1.0 * np.sum(itemRates.values()) / len(itemRates)\n",
    "    \n",
    "    upSum = 0.0\n",
    "    dwSum = 0.0\n",
    "    for b in users:\n",
    "        if b == userId:\n",
    "            continue\n",
    "        sim = self.sim_cal_UserCF(userId, b)\n",
    "        bRate = self.itemRatesForUser[b][eventId]\n",
    "        bMu = 1.0 * np.sum(self.itemRatesForUser[b].values()) / len(self.itemRatesForUser[b])\n",
    "        upSum += sim * (bRate - bMu)\n",
    "        dwSum += sim\n",
    "    \n",
    "    if dwSum == 0:\n",
    "        return 0.0\n",
    "    \n",
    "    ans = mu + 1.0 * upSum / dwSum\n",
    "    \n",
    "    if ans > 1:\n",
    "        return 1.0\n",
    "    elif ans < 0:\n",
    "        return 0.0\n",
    "    return ans\n",
    "\n",
    "\n",
    "  def sim_cal_ItemCF(self, i_id1, i_id2):\n",
    "    #计算Item i_id1和i_id2之间的相似性\n",
    "    #请补充完整代码\n",
    "    if i_id1 in self.itemSims and i_id2 in self.itemSims[i_id1]:\n",
    "        return self.itemSims[i_id1][i_id2]\n",
    "    \n",
    "    users1 = self.myUsersForItem[i_id1]\n",
    "    users2 = self.myUsersForItem[i_id2]\n",
    "    \n",
    "    upsum = 0\n",
    "    dwsum1 = 0\n",
    "    dwsum2 = 0\n",
    "    \n",
    "    for u in users1:\n",
    "        if u not in users2:\n",
    "            continue\n",
    "        r1 = self.itemRatesForUser[u][i_id1]\n",
    "        r2 = self.itemRatesForUser[u][i_id2]\n",
    "        mu = np.mean(self.itemRatesForUser[u].values())\n",
    "        upsum += (r1 - mu) * (r2 - mu)\n",
    "        dwsum1 += (r1 - mu) ** 2\n",
    "        dwsum2 += (r2 - mu) ** 2\n",
    "    \n",
    "    if dwsum1 == 0 or dwsum2 == 0:\n",
    "        self.itemSims.setdefault(i_id1, {})\n",
    "        self.itemSims.setdefault(i_id2, {})\n",
    "        self.itemSims[i_id1][i_id2] = 0.0\n",
    "        self.itemSims[i_id2][i_id1] = 0.0\n",
    "        return 0.0\n",
    "    \n",
    "    similarity = 1.0 * upsum / np.sqrt(dwsum1) * np.sqrt(dwsum2)\n",
    "    \n",
    "    self.itemSims.setdefault(i_id1, {})\n",
    "    self.itemSims.setdefault(i_id2, {})\n",
    "    self.itemSims[i_id1][i_id2] = similarity\n",
    "    self.itemSims[i_id2][i_id1] = similarity\n",
    "        \n",
    "    return similarity\n",
    "            \n",
    "  def eventCFReco(self, userId, eventId):    \n",
    "    \"\"\"\n",
    "    根据基于物品的协同过滤，得到Event的推荐度\n",
    "    基本的伪代码思路如下：\n",
    "    for item i \n",
    "        for every item j tht u has a preference for\n",
    "            compute similarity s between i and j\n",
    "            add u's preference for j weighted by s to a running average\n",
    "    return top items, ranked by weighted average\n",
    "    \"\"\"\n",
    "    #请补充完整代码\n",
    "    \n",
    "    if userId not in self.itemRatesForUser:\n",
    "        return 0.0\n",
    "    \n",
    "    itemRates = self.itemRatesForUser[userId]\n",
    "    \n",
    "    upsum = 0.0\n",
    "    dwsum = 0.0\n",
    "    \n",
    "    for i in itemRates:\n",
    "        r = itemRates[i]\n",
    "        sim = self.sim_cal_ItemCF(i, eventId)\n",
    "        upsum += sim * r\n",
    "        dwsum += sim\n",
    "    \n",
    "    if dwsum == 0:\n",
    "        return 0.0\n",
    "    \n",
    "    ans = 1.0 * upsum / dwsum\n",
    "    \n",
    "    if ans > 1:\n",
    "        return 1.0\n",
    "    elif ans < 0:\n",
    "        return 0.0\n",
    "    \n",
    "    return ans\n",
    "    \n",
    "  def svdCFReco(self, userId, eventId):\n",
    "    #基于模型的协同过滤, SVD++/LFM\n",
    "    #u = self.userIndex[userId]\n",
    "    #i = self.eventIndex[eventId]\n",
    "\n",
    "    return self.pred_SVD(userId,eventId)\n",
    "\n",
    "  def userReco(self, userId, eventId):\n",
    "    \"\"\"\n",
    "    类似基于User-based协同过滤，只是用户之间的相似度由用户本身的属性得到，计算event的推荐度\n",
    "    基本的伪代码思路如下：\n",
    "    for item i\n",
    "      for every other user v that has a preference for i\n",
    "        compute similarity s between u and v\n",
    "        incorporate v's preference for i weighted by s into running aversge\n",
    "    return top items ranked by weighted average\n",
    "    \"\"\"\n",
    "    i = self.userIndex[userId]\n",
    "    j = self.eventIndex[eventId]\n",
    "\n",
    "    vs = self.userEventScores[:, j]\n",
    "    sims = self.userSimMatrix[i, :]\n",
    "\n",
    "    prod = sims * vs\n",
    "\n",
    "    try:\n",
    "      return prod[0, 0] - self.userEventScores[i, j]\n",
    "    except IndexError:\n",
    "      return 0\n",
    "\n",
    "  def eventReco(self, userId, eventId):\n",
    "    \"\"\"\n",
    "    类似基于Item-based协同过滤，只是item之间的相似度由item本身的属性得到，计算Event的推荐度\n",
    "    基本的伪代码思路如下：\n",
    "    for item i \n",
    "      for every item j that u has a preference for\n",
    "        compute similarity s between i and j\n",
    "        add u's preference for j weighted by s to a running average\n",
    "    return top items, ranked by weighted average\n",
    "    \"\"\"\n",
    "    i = self.userIndex[userId]\n",
    "    j = self.eventIndex[eventId]\n",
    "    js = self.userEventScores[i, :]\n",
    "    psim = self.eventPropSim[:, j]\n",
    "    csim = self.eventContSim[:, j]\n",
    "    pprod = js * psim\n",
    "    cprod = js * csim\n",
    "    \n",
    "    pscore = 0\n",
    "    cscore = 0\n",
    "    try:\n",
    "      pscore = pprod[0, 0] - self.userEventScores[i, j]\n",
    "    except IndexError:\n",
    "      pass\n",
    "    try:\n",
    "      cscore = cprod[0, 0] - self.userEventScores[i, j]\n",
    "    except IndexError:\n",
    "      pass\n",
    "    return pscore, cscore\n",
    "\n",
    "  def userPop(self, userId):\n",
    "    \"\"\"\n",
    "    基于用户的朋友个数来推断用户的社交程度\n",
    "    主要的考量是如果用户的朋友非常多，可能会更倾向于参加各种社交活动\n",
    "    \"\"\"\n",
    "    if self.userIndex.has_key(userId):\n",
    "      i = self.userIndex[userId]\n",
    "      try:\n",
    "        return self.numFriends[0, i]\n",
    "      except IndexError:\n",
    "        return 0\n",
    "    else:\n",
    "      return 0\n",
    "\n",
    "  def friendInfluence(self, userId):\n",
    "    \"\"\"\n",
    "    朋友对用户的影响\n",
    "    主要考虑用户所有的朋友中，有多少是非常喜欢参加各种社交活动/event的\n",
    "    用户的朋友圈如果都积极参与各种event，可能会对当前用户有一定的影响\n",
    "    \"\"\"\n",
    "    nusers = np.shape(self.userFriends)[1]\n",
    "    i = self.userIndex[userId]\n",
    "    return (self.userFriends[i, :].sum(axis=0) / nusers)[0,0]\n",
    "\n",
    "  def eventPop(self, eventId):\n",
    "    \"\"\"\n",
    "    本活动本身的热度\n",
    "    主要是通过参与的人数来界定的\n",
    "    \"\"\"\n",
    "    i = self.eventIndex[eventId]\n",
    "    return self.eventPopularity[i, 0]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVD Train...\n",
      "the  0 -th  step is running\n",
      "the  1 -th  step is running\n",
      "the  2 -th  step is running\n",
      "the  3 -th  step is running\n",
      "the  4 -th  step is running\n",
      "the  5 -th  step is running\n",
      "the  6 -th  step is running\n",
      "the  7 -th  step is running\n",
      "the  8 -th  step is running\n",
      "the  9 -th  step is running\n",
      "the  10 -th  step is running\n",
      "the  11 -th  step is running\n",
      "the  12 -th  step is running\n",
      "the  13 -th  step is running\n",
      "the  14 -th  step is running\n",
      "the  15 -th  step is running\n",
      "the  16 -th  step is running\n",
      "the  17 -th  step is running\n",
      "the  18 -th  step is running\n",
      "the  19 -th  step is running\n",
      "the  20 -th  step is running\n",
      "the  21 -th  step is running\n",
      "the  22 -th  step is running\n",
      "the  23 -th  step is running\n",
      "the  24 -th  step is running\n",
      "the  25 -th  step is running\n",
      "the  26 -th  step is running\n",
      "the  27 -th  step is running\n",
      "the  28 -th  step is running\n",
      "the  29 -th  step is running\n",
      "the  30 -th  step is running\n",
      "the  31 -th  step is running\n",
      "the  32 -th  step is running\n",
      "the  33 -th  step is running\n",
      "the  34 -th  step is running\n",
      "the  35 -th  step is running\n",
      "the  36 -th  step is running\n",
      "the  37 -th  step is running\n",
      "the  38 -th  step is running\n",
      "the  39 -th  step is running\n",
      "the  40 -th  step is running\n",
      "the  41 -th  step is running\n",
      "the  42 -th  step is running\n",
      "the  43 -th  step is running\n",
      "the  44 -th  step is running\n",
      "the  45 -th  step is running\n",
      "the  46 -th  step is running\n",
      "the  47 -th  step is running\n",
      "the  48 -th  step is running\n",
      "the  49 -th  step is running\n",
      "the  50 -th  step is running\n",
      "the  51 -th  step is running\n",
      "the  52 -th  step is running\n",
      "the  53 -th  step is running\n",
      "the  54 -th  step is running\n",
      "the  55 -th  step is running\n",
      "the  56 -th  step is running\n",
      "the  57 -th  step is running\n",
      "the  58 -th  step is running\n",
      "the  59 -th  step is running\n",
      "the  60 -th  step is running\n",
      "the  61 -th  step is running\n",
      "the  62 -th  step is running\n",
      "the  63 -th  step is running\n",
      "the  64 -th  step is running\n",
      "the  65 -th  step is running\n",
      "the  66 -th  step is running\n",
      "the  67 -th  step is running\n",
      "the  68 -th  step is running\n",
      "the  69 -th  step is running\n",
      "the  70 -th  step is running\n",
      "the  71 -th  step is running\n",
      "the  72 -th  step is running\n",
      "the  73 -th  step is running\n",
      "the  74 -th  step is running\n",
      "the  75 -th  step is running\n",
      "the  76 -th  step is running\n",
      "the  77 -th  step is running\n",
      "the  78 -th  step is running\n",
      "the  79 -th  step is running\n",
      "the  80 -th  step is running\n",
      "the  81 -th  step is running\n",
      "the  82 -th  step is running\n",
      "the  83 -th  step is running\n",
      "the  84 -th  step is running\n",
      "the  85 -th  step is running\n",
      "the  86 -th  step is running\n",
      "the  87 -th  step is running\n",
      "the  88 -th  step is running\n",
      "the  89 -th  step is running\n",
      "the  90 -th  step is running\n",
      "the  91 -th  step is running\n",
      "the  92 -th  step is running\n",
      "the  93 -th  step is running\n",
      "the  94 -th  step is running\n",
      "the  95 -th  step is running\n",
      "the  96 -th  step is running\n",
      "the  97 -th  step is running\n",
      "the  98 -th  step is running\n",
      "the  99 -th  step is running\n",
      "SVD trained\n"
     ]
    }
   ],
   "source": [
    "# 初始化与训练推荐系统\n",
    "rs = RecommonderSystem(trainfile=\"data/train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rmse of test userCF is  0.4835914845822187\n",
      "rmse of test itemCF is  0.6382944005425294\n"
     ]
    }
   ],
   "source": [
    "# 测试与RMSE评估\n",
    "from __future__ import division\n",
    "\n",
    "import cPickle\n",
    "import numpy as np\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "from numpy.random import random  \n",
    "from collections import defaultdict\n",
    "\n",
    "import pandas as pd \n",
    "\n",
    "def test_svd(test_X):\n",
    "    n = test_X.shape[0]\n",
    "    mse = 0\n",
    "    for x in test_X:\n",
    "        mse += (x[4] - rs.pred_SVD(x[0], x[1]))**2\n",
    "    rmse = np.sqrt(mse/n)\n",
    "    return rmse\n",
    "\n",
    "def test_user_CF(test_X):\n",
    "    n = test_X.shape[0]\n",
    "    mse = 0\n",
    "    for x in test_X:\n",
    "        preditScore = rs.userCFReco(x[0], x[1])\n",
    "        mse += (x[4] - preditScore)**2\n",
    "    rmse = np.sqrt(mse/n)\n",
    "    return rmse\n",
    "\n",
    "def test_item_CF(test_X):\n",
    "    n = test_X.shape[0]\n",
    "    mse = 0\n",
    "    for x in test_X:\n",
    "        preditScore = rs.eventCFReco(x[0], x[1])\n",
    "        mse += (x[4] - preditScore)**2\n",
    "    rmse = np.sqrt(mse/n)\n",
    "    return rmse\n",
    "\n",
    "test_data = pd.read_csv(\"data/train.csv\")\n",
    "test_X = np.array(test_data)\n",
    "\n",
    "r_svd = test_svd(test_X)\n",
    "print \"rmse of test svd is \",r_svd\n",
    "\n",
    "r_user_CF = test_user_CF(test_X)\n",
    "print \"rmse of test userCF is \",r_user_CF\n",
    "\n",
    "r_item_CF = test_item_CF(test_X)\n",
    "print \"rmse of test itemCF is \",r_item_CF\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generateRSData(RS, fn, train=True, header=True):\n",
    "    \"\"\"\n",
    "    把前面user-based协同过滤 和 item-based协同过滤，以及各种热度和影响度作为特征组合在一起\n",
    "    生成新的训练数据，用于分类器分类使用\n",
    "    \"\"\"\n",
    "    fin = open(fn, 'rb')\n",
    "    fout = open(\"RS_\" + fn, 'wb')\n",
    "    \n",
    "    #忽略第一行（列名字）\n",
    "    fin.readline().strip().split(\",\")\n",
    "    \n",
    "    # write output header\n",
    "    if header:\n",
    "      ocolnames = [\"invited\", \"userCF_reco\", \"evtCF_reco\",\"svdCF_reco\",\"user_reco\", \"evt_p_reco\",\n",
    "        \"evt_c_reco\", \"user_pop\", \"frnd_infl\", \"evt_pop\"]\n",
    "      if train:\n",
    "        ocolnames.append(\"interested\")\n",
    "        ocolnames.append(\"not_interested\")\n",
    "      fout.write(\",\".join(ocolnames) + \"\\n\")\n",
    "    \n",
    "    ln = 0\n",
    "    for line in fin:\n",
    "      ln += 1\n",
    "      if ln%500 == 0:\n",
    "          print \"%s:%d (userId, eventId)=(%s, %s)\" % (fn, ln, userId, eventId)\n",
    "          #break;\n",
    "      \n",
    "      cols = line.strip().split(\",\")\n",
    "      userId = cols[0]\n",
    "      eventId = cols[1]\n",
    "      invited = cols[2]\n",
    "        \n",
    "      userCF_reco = RS.userCFReco(int(userId), int(eventId))\n",
    "      itemCF_reco = RS.eventCFReco(int(userId), int(eventId))\n",
    "      svdCF_reco = RS.svdCFReco(int(userId), int(eventId))\n",
    "        \n",
    "      user_reco = RS.userReco(userId, eventId)\n",
    "      evt_p_reco, evt_c_reco = RS.eventReco(userId, eventId)\n",
    "      user_pop = RS.userPop(userId)\n",
    "     \n",
    "      frnd_infl = RS.friendInfluence(userId)\n",
    "      evt_pop = RS.eventPop(eventId)\n",
    "      ocols = [invited, userCF_reco, itemCF_reco, svdCF_reco,user_reco, evt_p_reco,\n",
    "        evt_c_reco, user_pop, frnd_infl, evt_pop]\n",
    "      \n",
    "      if train:\n",
    "        ocols.append(cols[4]) # interested\n",
    "        ocols.append(cols[5]) # not_interested\n",
    "      fout.write(\",\".join(map(lambda x: str(x), ocols)) + \"\\n\")\n",
    "    \n",
    "    fin.close()\n",
    "    fout.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVD Train...\n",
      "the  0 -th  step is running\n",
      "the  1 -th  step is running\n",
      "the  2 -th  step is running\n",
      "the  3 -th  step is running\n",
      "the  4 -th  step is running\n",
      "the  5 -th  step is running\n",
      "the  6 -th  step is running\n",
      "the  7 -th  step is running\n",
      "the  8 -th  step is running\n",
      "the  9 -th  step is running\n",
      "the  10 -th  step is running\n",
      "the  11 -th  step is running\n",
      "the  12 -th  step is running\n",
      "the  13 -th  step is running\n",
      "the  14 -th  step is running\n",
      "the  15 -th  step is running\n",
      "the  16 -th  step is running\n",
      "the  17 -th  step is running\n",
      "the  18 -th  step is running\n",
      "the  19 -th  step is running\n",
      "the  20 -th  step is running\n",
      "the  21 -th  step is running\n",
      "the  22 -th  step is running\n",
      "the  23 -th  step is running\n",
      "the  24 -th  step is running\n",
      "the  25 -th  step is running\n",
      "the  26 -th  step is running\n",
      "the  27 -th  step is running\n",
      "the  28 -th  step is running\n",
      "the  29 -th  step is running\n",
      "the  30 -th  step is running\n",
      "the  31 -th  step is running\n",
      "the  32 -th  step is running\n",
      "the  33 -th  step is running\n",
      "the  34 -th  step is running\n",
      "the  35 -th  step is running\n",
      "the  36 -th  step is running\n",
      "the  37 -th  step is running\n",
      "the  38 -th  step is running\n",
      "the  39 -th  step is running\n",
      "the  40 -th  step is running\n",
      "the  41 -th  step is running\n",
      "the  42 -th  step is running\n",
      "the  43 -th  step is running\n",
      "the  44 -th  step is running\n",
      "the  45 -th  step is running\n",
      "the  46 -th  step is running\n",
      "the  47 -th  step is running\n",
      "the  48 -th  step is running\n",
      "the  49 -th  step is running\n",
      "the  50 -th  step is running\n",
      "the  51 -th  step is running\n",
      "the  52 -th  step is running\n",
      "the  53 -th  step is running\n",
      "the  54 -th  step is running\n",
      "the  55 -th  step is running\n",
      "the  56 -th  step is running\n",
      "the  57 -th  step is running\n",
      "the  58 -th  step is running\n",
      "the  59 -th  step is running\n",
      "the  60 -th  step is running\n",
      "the  61 -th  step is running\n",
      "the  62 -th  step is running\n",
      "the  63 -th  step is running\n",
      "the  64 -th  step is running\n",
      "the  65 -th  step is running\n",
      "the  66 -th  step is running\n",
      "the  67 -th  step is running\n",
      "the  68 -th  step is running\n",
      "the  69 -th  step is running\n",
      "the  70 -th  step is running\n",
      "the  71 -th  step is running\n",
      "the  72 -th  step is running\n",
      "the  73 -th  step is running\n",
      "the  74 -th  step is running\n",
      "the  75 -th  step is running\n",
      "the  76 -th  step is running\n",
      "the  77 -th  step is running\n",
      "the  78 -th  step is running\n",
      "the  79 -th  step is running\n",
      "the  80 -th  step is running\n",
      "the  81 -th  step is running\n",
      "the  82 -th  step is running\n",
      "the  83 -th  step is running\n",
      "the  84 -th  step is running\n",
      "the  85 -th  step is running\n",
      "the  86 -th  step is running\n",
      "the  87 -th  step is running\n",
      "the  88 -th  step is running\n",
      "the  89 -th  step is running\n",
      "the  90 -th  step is running\n",
      "the  91 -th  step is running\n",
      "the  92 -th  step is running\n",
      "the  93 -th  step is running\n",
      "the  94 -th  step is running\n",
      "the  95 -th  step is running\n",
      "the  96 -th  step is running\n",
      "the  97 -th  step is running\n",
      "the  98 -th  step is running\n",
      "the  99 -th  step is running\n",
      "SVD trained\n",
      "生成训练数据...\n",
      "\n",
      "data/train.csv:500 (userId, eventId)=(123290209, 1887085024)\n",
      "data/train.csv:1000 (userId, eventId)=(272886293, 199858305)\n",
      "data/train.csv:1500 (userId, eventId)=(395305791, 1582270949)\n",
      "data/train.csv:2000 (userId, eventId)=(527523423, 3272728211)\n",
      "data/train.csv:2500 (userId, eventId)=(651258472, 792632006)\n",
      "data/train.csv:3000 (userId, eventId)=(811791433, 524756826)\n",
      "data/train.csv:3500 (userId, eventId)=(985547042, 1269035551)\n",
      "data/train.csv:4000 (userId, eventId)=(1107615001, 173949238)\n",
      "data/train.csv:4500 (userId, eventId)=(1236336671, 3849306291)\n",
      "data/train.csv:5000 (userId, eventId)=(1414301782, 2652356640)\n",
      "data/train.csv:5500 (userId, eventId)=(1595465532, 955398943)\n",
      "data/train.csv:6000 (userId, eventId)=(1747091728, 2131379889)\n",
      "data/train.csv:6500 (userId, eventId)=(1914182220, 955398943)\n",
      "data/train.csv:7000 (userId, eventId)=(2071842684, 1076364848)\n",
      "data/train.csv:7500 (userId, eventId)=(2217853337, 3051438735)\n",
      "data/train.csv:8000 (userId, eventId)=(2338481531, 2525447278)\n",
      "data/train.csv:8500 (userId, eventId)=(2489551967, 520657921)\n",
      "data/train.csv:9000 (userId, eventId)=(2650493630, 87962584)\n",
      "data/train.csv:9500 (userId, eventId)=(2791418962, 4223848259)\n",
      "data/train.csv:10000 (userId, eventId)=(2903662804, 2791462807)\n",
      "data/train.csv:10500 (userId, eventId)=(3036141956, 3929507420)\n",
      "data/train.csv:11000 (userId, eventId)=(3176074542, 3459485614)\n",
      "data/train.csv:11500 (userId, eventId)=(3285425249, 2271782630)\n",
      "data/train.csv:12000 (userId, eventId)=(3410667855, 1063772489)\n",
      "data/train.csv:12500 (userId, eventId)=(3531604778, 2584839423)\n",
      "data/train.csv:13000 (userId, eventId)=(3686871863, 53495098)\n",
      "data/train.csv:13500 (userId, eventId)=(3833637800, 2415873572)\n",
      "data/train.csv:14000 (userId, eventId)=(3944021305, 2096772901)\n",
      "data/train.csv:14500 (userId, eventId)=(4075466480, 3567240505)\n",
      "data/train.csv:15000 (userId, eventId)=(4197193550, 1628057176)\n",
      "生成预测数据...\n",
      "\n",
      "data/test.csv:500 (userId, eventId)=(182290053, 2529072432)\n",
      "data/test.csv:1000 (userId, eventId)=(433510318, 4244463632)\n",
      "data/test.csv:1500 (userId, eventId)=(632808865, 2845303452)\n",
      "data/test.csv:2000 (userId, eventId)=(813611885, 2036538169)\n",
      "data/test.csv:2500 (userId, eventId)=(1010701404, 303459881)\n",
      "data/test.csv:3000 (userId, eventId)=(1210932037, 2529072432)\n",
      "data/test.csv:3500 (userId, eventId)=(1452921099, 2705317682)\n",
      "data/test.csv:4000 (userId, eventId)=(1623287180, 1626678328)\n",
      "data/test.csv:4500 (userId, eventId)=(1855201342, 2603032829)\n",
      "data/test.csv:5000 (userId, eventId)=(2083900381, 2529072432)\n",
      "data/test.csv:5500 (userId, eventId)=(2318415276, 2509151803)\n",
      "data/test.csv:6000 (userId, eventId)=(2528161539, 4025975316)\n",
      "data/test.csv:6500 (userId, eventId)=(2749110768, 4244406355)\n",
      "data/test.csv:7000 (userId, eventId)=(2927772127, 1532377761)\n",
      "data/test.csv:7500 (userId, eventId)=(3199685636, 1776393554)\n",
      "data/test.csv:8000 (userId, eventId)=(3393388475, 680270887)\n",
      "data/test.csv:8500 (userId, eventId)=(3601169721, 154434302)\n",
      "data/test.csv:9000 (userId, eventId)=(3828963415, 3067222491)\n",
      "data/test.csv:9500 (userId, eventId)=(4018723397, 2522610844)\n",
      "data/test.csv:10000 (userId, eventId)=(4180064266, 2658555390)\n"
     ]
    }
   ],
   "source": [
    "train_fn = \"data/train.csv\"\n",
    "RS = RecommonderSystem(train_fn)\n",
    "print \"生成训练数据...\\n\"\n",
    "generateRSData(RS, train_fn, train=True,  header=True)\n",
    "\n",
    "test_fn = \"data/test.csv\"\n",
    "print \"生成预测数据...\\n\"\n",
    "generateRSData(RS, test_fn, train=False, header=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "时间、地点等特征都没有处理了，可以考虑用户看到event的时间与event开始时间的差、用户地点和event地点的差异。。。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
